<?php

/*
 *  This file is part of Urd.
 *
 *  Urd is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 3 of the License, or
 *  (at your option) any later version.
 *  Urd is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program. See the file "COPYING". If it does not 
 *  exist, see <http://www.gnu.org/licenses/>.
 * 
 * $LastChangedDate: 2009-01-04 14:49:53 +0100 (Sun, 04 Jan 2009) $
 * $Rev: 411 $
 * $Author: gavinspearhead $
 * $Id: download_functions.php 411 2009-01-04 13:49:53Z gavinspearhead $
 */

if (!defined('ORIGINAL_PAGE')) die('This file cannot be accessed directly.');

$pathdlf = realpath(dirname(__FILE__));


require_once "$pathdlf/../functions/autoincludes.php";
require_once "$pathdlf/../functions/urdversion.php";
require_once "$pathdlf/../functions/defines.php";
require_once "$pathdlf/../config.php";
require_once "$pathdlf/../functions/functions.php";
require_once "$pathdlf/../functions/pref_functions.php";
require_once "$pathdlf/urdd_command.php";
require_once "$pathdlf/urdd_protocol.php";
require_once "$pathdlf/urdd_error.php";
require_once "$pathdlf/../functions/urd_log.php";


function store_article($article, $dir, $msg_id)
{
    $msg_id = trim($msg_id, '<>');
    $article = implode($article, "\n");
    file_put_contents($dir . $msg_id . '.txt', $article. "\n\n");
}


function get_download_type(array &$article)
{
    $possible_xxencode = $possible_uuencode = FALSE;
    $cnt = 0;
    foreach ($article as $line)	{
        if (!isset($line[0])) continue;
        if (isset($line[1]) && substr_compare($line, '=y', 0, 2) == 0) {
            return TYPE_YYENCODED;
        } else if ($line[0] == 'M') {
            if ($possible_uuencode === TRUE) 
                return TYPE_UUENCODED; 
            else 
                $possible_uuencode = TRUE;
        } else if ($line[0] == 'h') {
            // if it is 'h' it is XXencoded... do we need that? does anyone use XXencode?
            if ($possible_xxencode === TRUE) 
                return TYPE_XXENCODED; 
            else 
                $possible_xxencode = TRUE;
        } else if ($cnt < 5 && stripos ($line, 'This is a multipart message in MIME format.') !== FALSE) {
            return TYPE_MIMEENCODED;
        } // else dunno - need to look further
        ++$cnt;
        if ($cnt > MAX_ART_LINE_COUNT)
            break;
    }
    return TYPE_UNKNOWN;
}


function check_for_encrypted_rar($art, $dir)
{
    static $chunk_buffer = array();
    $buffer_size = RARSCAN_QUEUESIZE * 3; 

    $cnt = 0;
    $filechunkstart = NULL;
    $filechunkend   = NULL;
    $filechunksize  = NULL;
    $filename       = NULL;

    // get filename, offset and chunk size from article
    foreach ($art as $line)	{
        if (!isset($line[0])) continue;
        if (isset($line[1]) && substr_compare($line, '=y', 0, 2) == 0) {
            if (preg_match('/name=(.*)$/', $line, $matches))
                $filename = $matches[1];
            if (preg_match('/begin=(\d+)/', $line, $matches))
                $filechunkstart = ($matches[1]) - 1; // yenc vars not 0-based
            if (preg_match('/end=(\d+)/', $line, $matches))
                $filechunkend = ($matches[1]) - 1; // yenc vars not 0-based

            if (($filename !== NULL) && ($filechunkstart !== NULL) && ($filechunkend !== NULL)) 
                break;
            if (++$cnt > MAX_ART_LINE_COUNT){
                echo_debug('Couldn\'t locate needed chunk data in article header to analyze block', DEBUG_SERVER);
                return FALSE;
            }
        } 
    }

    $filechunksize = $filechunkend - $filechunkstart;
    $fileoffset = $filechunkstart - 50; // Try to read 50 chars before current block to catch headers over split articles
    if ($fileoffset < 0) {
        $fileoffset = 0;
    }
    $bufcnt = array_push($chunk_buffer, ($dir . $filename), $fileoffset, $filechunksize); // Save chunk metadata last in queue buffer for later use
    if ($bufcnt < $buffer_size) {  // We will not start analyzing anything until buffer is big enough - yydecode thread may not have had time to write to disk yet
        return FALSE;
    }

    // Now we read the oldest record from the buffer and process it - yydecode should have had time to write do disk by now.
    $filename      = array_shift($chunk_buffer);
    $fileoffset    = array_shift($chunk_buffer);
    $filechunksize = array_shift($chunk_buffer);
    $data = file_get_contents($filename, FALSE, NULL, $fileoffset, $filechunksize + 50); // OK to add compensating 50 since it won't read past EOF anyway
    $chunksizeread = strlen($data);
    if ($chunksizeread < $filechunksize) {
        echo_debug("Unexpected chunk size read: $chunksizeread, expected (at least) $filechunksize. File: $filename Offset: $fileoffset", DEBUG_SERVER);
        echo_debug("If you see this, try increasing RARSCAN_QUEUESIZE slightly since it may mean yydecode is falling behind", DEBUG_SERVER);
        return FALSE;
    } else {
        echo_debug("Data chunk read OK. File: $filename Offset: $fileoffset Read chunk size: $chunksizeread", DEBUG_SERVER);
    }
    // Loop through any "Rar!" matches in read data and look for encryption flags if header found 
    $dataoffset = strpos($data, 'Rar!');
    while($dataoffset !== FALSE) {
        echo_debug("Rar! header found at data offset $dataoffset", DEBUG_SERVER);
        if ($dataoffset >= $filechunksize - 25) { // Safety - we may end up reading memory past end of string. We'll catch this next round instead
            return FALSE;
        }
        if ((ord($data[$dataoffset + 10]) & 0x80) === 0x80) {
            echo_debug("Rar! Block encryption detected!", DEBUG_SERVER);
            return TRUE; 
        }
        if ((ord($data[$dataoffset + 23]) & 0x04) === 0x04) { 
            echo_debug("Rar! File encryption detected!", DEBUG_SERVER);
            return TRUE; 
        }
        $dataoffset = strpos($data, 'Rar!', $dataoffset + 4); // Continue searching right after last "Rar!", there may be more. Also, fake Rar! headers won't fool us :P
    }
    return FALSE; // No encryption flags found if we get here
}


function download_batch(DatabaseConnection $db, array &$batch, $dir, URD_NNTP &$nzb, &$groupid, $username, &$connected, $check_for_rar_encryption)
{
	echo_debug_function(DEBUG_SERVER, __FUNCTION__);
	$yydecode = my_escapeshellcmd (get_config($db, 'yydecode_path'));
	$yydecode_pars = my_escapeshellarg(get_config_default($db, 'yydecode_pars', ''), FALSE);
	if ($yydecode === FALSE)
		throw new exception('yydecode not found', ERR_CONFIG_ERROR);

	$cmd = "/bin/sh -c '$yydecode $yydecode_pars ' ";
	$size = $a_cnt = $e_cnt = $groupid = 0;

	static $mime_settings = array('include_bodies'=>TRUE, 'decode_bodies'=>TRUE, 'decode_headers'=>TRUE);
	static $descriptorspec = array ( 
			0 => array('pipe', 'r'), // where we will write to
			1 => array('file', '/dev/null', 'w'), // we don't want the output
			2 => array('file', '/dev/null', 'w') // or the errors
			//	1 => array('file', '/tmp/out', 'a'), // we don't want the output
			//		2 => array('file', '/tmp/err', 'a') // or the errors
			);
	$pipes = array();

	$process = proc_open($cmd, $descriptorspec, $pipes, $dir, NULL, array('binary_pipes'));
	if ($process === FALSE || !is_resource($process)) {
		write_log('Could not create pipe', LOG_WARNING);
		exit (NNTP_NOT_CONNECTED_ERROR);
    }
    $download_text_file = get_pref($db, 'download_text_file', $username);

	$p = $pipes[0];

	// $batch is the result of the select query on ready-articles:
	// First reset them all to DOWNLOAD_READY, because if we throw an error we don't want to have orphaned ACTIVE's
	
	foreach ($batch as $key => $article) {
		try {
			if ($article['groupID'] != $groupid) {
				$groupid = $article['groupID'];
				$data = $nzb->select_group($groupid, $code);
			}

			$msg_id = '<' . $article['messageID'] . '>';
			$art = $nzb->get_article($msg_id);
			// If we get here, download was succesful (otherwise try/catch kicked in)
            $batch[$key]['dlstatus'] = DOWNLOAD_FINISHED;
            $partnr = NULL;
			$type = get_download_type($art);

			// Check type of encoding:
			if ($type == TYPE_YYENCODED) {
				$err_level = error_reporting(0); // so we don't get any write errors, saves a @
				foreach($art as $line) {
					$r = fwrite($p, $line . "\n");
					if ($r === FALSE) {
						error_reporting($err_level);
						throw new exception('Write failed', ERR_PIPE_ERROR);
					}
					$size += $r;
                }
                if ($check_for_rar_encryption) {
                    $is_enc = check_for_encrypted_rar($art, $dir);
                    if ($is_enc) {
                        touch($dir . '/password encrypted file.log');
                        throw new exception ('Password encrypted file - cancelling', ENCRYPTED_RAR);
                    }
                }
				error_reporting($err_level);
				$a_cnt++;
			} elseif ($type == TYPE_UUENCODED) {
				$partnumber = $article['partnumber'];
				$name = preg_replace('/[^a-zA-Z0-9._]/', '', $article['name']);
				$f = fopen($dir . $name . '.' . $partnumber. '.urd_uuencoded_part', 'w');
				if($f === FALSE)
					throw new exception('failed to create file', ERR_PIPE_ERROR);
				$err_level = error_reporting(0); // so we don't get any write errors, saves a @
				foreach($art as $line) {
					$r = fwrite($f, $line . "\n");
					if ($r === FALSE) {
						error_reporting($err_level);
						throw new exception('write failed', ERR_PIPE_ERROR);
					}
					$size += $r;
				}
				fclose($f);
				error_reporting($err_level);
				$a_cnt++;
			} elseif ($type == TYPE_XXENCODED) {
				write_log('Can not handle XX encoded files yet... post at the forum if you run in to this error', LOG_ERR);
				throw new exception('Unrecognized encoding found', ERR_UNKNOWN_ENCODING);
			} elseif ($type == TYPE_MIMEENCODED) {
				$head = $nzb->get_header($msg_id);
				$mime = new Mail_mimeDecode($head,  $art);
				$mime_settings = array('include_bodies'=>TRUE, 'decode_bodies'=>TRUE, 'decode_headers'=>TRUE);
				$res = $mime->decode($mime_settings);
				foreach ($res->parts as $part) {
					if (isset($part->d_parameters['filename'])) {
						$filename = $dir . $part->d_parameters['filename'];
						file_put_contents($filename, $part->body);
					}
					$size += count($part->body);
				}
			} else {
				if ($download_text_file && count($art) < (4 * 1024))
					store_article($art, $dir, $msg_id);
				else 
					throw new exception('Unrecognized encoding found', ERR_UNKNOWN_ENCODING);
			}
		} catch(exception $e) {
			$e_cnt ++;
			$batch[$key]['dlstatus'] = DOWNLOAD_FAILED;
			write_log('Could not download article: ' . $e->getMessage() ."({$e->getCode()})", LOG_NOTICE);

			if ($e->getCode() == NNTP_NOT_CONNECTED_ERROR) {
				// Articles didn't really fail so set them to be downloaded again:
				$batch[$key]['dlstatus'] = DOWNLOAD_READY;
				pclose($p);
				$rv = proc_close($process);
				// Set connected to false so start_download function knows it shouldn't continue
				$connected = FALSE;
				// And return because we don't want to continue trying to download stuff from this batch:
				return array($size,$a_cnt,$e_cnt);
//				exit (NNTP_NOT_CONNECTED_ERROR);
			}
            else if ($e->getCode() == ENCRYPTED_RAR) {
                throw $e;
            }
		}
	}
	if ($a_cnt > 0) write_log("Downloaded $a_cnt articles", LOG_INFO);
	if ($e_cnt > 0) write_log("Failed to download $e_cnt articles", LOG_INFO);
	pclose($p);
	$rv = proc_close($process);
	if ($rv > 1)
		write_log("YYdecode was not successful (exit code $rv) - disk full?", LOG_WARNING);

	// Should update download status here, errors/completed/todo/in progress XXX XXX XXX
	return array($size, $a_cnt, $e_cnt);
}


function start_download(action $item, DatabaseConnection $db)
{
	$groupid = 0;
	$dlid = $item->get_args();
	if (check_dl_lock($db, $dlid) === FALSE) { // if db still locked
		echo_debug('Dl still locked, sleeping', DEBUG_SERVER); // todo needs fixing
		return DB_LOCKED;
	}
	$stat_id = get_stat_id($db, $dlid);

	$dbid = $item->get_dbid();
	$total_ready = get_download_articles_count_status($db, $dlid, DOWNLOAD_READY);
    $total = get_download_articles_count($db, $dlid);
    $dlpw = get_download_password($db, $dlid) ? TRUE : FALSE;
    $username = $item->get_username();
    $check_for_rar_encryption = get_pref($db, 'cancel_crypted_rars', $username, FALSE);
    if ($dlpw != '')
        $check_for_rar_encryption = FALSE;
	$done_start = $total - $total_ready;
	mt_srand(getmypid());
	// determine a reasonable batchsize... increase it if we have more articles
	if ($item->get_preview() === TRUE)
		$batch_size = PV_BATCH_SIZE;
	else  if ($total_ready < (10 * DL_BATCH_SIZE))
		$batch_size = DL_BATCH_SIZE + mt_rand(1, 4);
	else  if ($total_ready < (20 * DL_BATCH_SIZE))
		$batch_size = (2 * DL_BATCH_SIZE) + mt_rand(1, DL_BATCH_SIZE);
	else
		$batch_size = (4 * DL_BATCH_SIZE) + mt_rand(1, DL_BATCH_SIZE);
	$done = $cnt = 0;
	$first_batch_size = max(4, DL_BATCH_SIZE / 4); // the first batch is always small,  so we get a quick progress update
	$dir = get_download_destination($db, $dlid);
	$start_time = get_start_time($db, $dlid);	
	if ($start_time > time())
		set_start_time($db, $dlid, time());

	static $req_status = DOWNLOAD_READY;
	static $dl_status = DOWNLOAD_ACTIVE;
	static $done_status = DOWNLOAD_FINISHED;
	static $failed_status = DOWNLOAD_FAILED;

	// Update status:
	update_dlinfo_status($db, $dl_status, $dlid);
	static $lock_array = array('downloadarticles'=>'write'); // for the article table

	try {
		// Pick a server:
		$server_id = $item->get_active_server();
		if ($server_id == 0) 
			$server_id = $item->get_preferred_server();

		// Connect
		$reconnects = 0;
		$nzb = NULL;

		$b_time = microtime(TRUE);
		$connected = FALSE;
		while (TRUE) {
			if ($connected === FALSE) {
				// Check if we didn't exceed max allowed reconnect attempts:
                if ($reconnects >= MAX_RECONNECTION_ATTEMPTS_PER_THREAD) 
                    throw new exception('Max reconnection attempts exceeded'); 

				// Connect:
				write_log('Connecting to NNTP server.', LOG_NOTICE);
				$nzb = connect_nntp($db, $server_id);
				$connected = TRUE;
				$reconnects++;
			}

			$s_time = microtime(TRUE);
			$db->lock($lock_array);
			try {
				$query = "* FROM downloadarticles WHERE \"downloadID\"=$dlid AND \"status\"=$req_status ORDER BY name, partnumber";
				// First time use small batch size:
				if ($first_batch_size != 0) {
					$res = $db->select_query($query, $first_batch_size);
					$first_batch_size = 0;
				} else {
					$res = $db->select_query($query, $batch_size);
				}

				if ($res === FALSE) {
					// Good exit:
					$db->unlock();
					echo_debug('No more articles found', DEBUG_NNTP);
					$nzb->disconnect();
					$comment = "Processed $cnt batches";
					write_log($comment, LOG_DEBUG);
					$status = QUEUE_FINISHED;
					$progress = NULL; // not set the progress... other threads may still be running....
					$error_no = NO_ERROR;
					update_queue_status($db, $item->get_dbid(), $status, 0, $progress, $comment);
					return $error_no;
				}

				// Set all articles from $res to the DOWNLOAD_ACTIVE status:
				update_batch($db, $res, $dl_status);
			} catch (exception $e) {
				$db->unlock();
				$nzb->disconnect();
				throw $e;
			}
			$db->unlock();

            // Download the batch:
            try {
                list ($bytes, $a_cnt, $e_cnt) = download_batch($db, $res, $dir, $nzb, $groupid, $username, $connected, $check_for_rar_encryption);
            } catch (exception $e) {
                if ($e->getCode() == ENCRYPTED_RAR) {
                    $status = QUEUE_CANCELLED;
                    $progress = 0;
                    $comment = $e->getMessage();
                    write_log('Cancelling download: ' . $e->getmessage(), LOG_NOTICE);
                    update_queue_status($db, $item->get_dbid(), $status, 0, $progress, $comment);
                    exit(ENCRYPTED_RAR);
                } else 
                    throw $e;
            }

			// Downloading changed the status of some articles, update database with new statuses:
			// But before we do that, we can't have any batch items on active as the batch is finished:
			// (this can happen when the NNTP server disconnects:
			if ($connected == FALSE) {
				foreach ($res as $key => $article) {
					if (!isset($res[$key]['dlstatus']) || (isset($res[$key]['dlstatus']) && $res[$key]['dlstatus'] == $dl_status)) 
						$res[$key]['dlstatus'] = $req_status;
				}
			}

			// $req_status means that the default status is 'DOWNLOAD_READY', if for some reason,
			// some batch items won't have a specified status, and then $req_status is the default value
			update_batch($db, $res, $req_status);
			// Download statistics:
			update_dlstats($db, $stat_id, $bytes);
			$f_time = microtime(TRUE);
			$time_diff = $f_time - $b_time;
			$cnt++;
			$done = get_download_articles_count_status($db, $dlid, $done_status);
			$failed = get_download_articles_count_status($db, $dlid, $failed_status);
			$percentage = ($total > 0 ) ? floor(100 * ($done / $total)) : 0;
			$remain = $total - $done;
			$done_ready = $done - $done_start;
			$eta = ($done_ready > 0) ? (round(($remain * $time_diff) / $done_ready)) : 0;
			$speed = ($time_diff > 0) ? (round($bytes / ($time_diff))) : 0;
			store_ETA($db, $eta, $percentage, $speed, $dbid);
		}
	} catch (exception $e) {
		write_log('Error while downloading: ' . $e->getmessage(), LOG_NOTICE);
	}

	// Bad exit:
	$status = QUEUE_FAILED;
	$comment = 'Connection failed';
	$progress = 0;
	$error_no = NNTP_NOT_CONNECTED_ERROR;
	update_queue_status($db, $item->get_dbid(), $status, 0, $progress, $comment);
	return $error_no;
}


function complete_download(DatabaseConnection $db, server_data &$servers, action $item, $status)
{
    echo_debug_function(DEBUG_SERVER, __FUNCTION__);
    if (!$servers->has_equal($item)) { 
        echo_debug("last download (STATUS $status)", DEBUG_SERVER);
        $dlid = $item->get_args();
        if ($status == DOWNLOAD_FINISHED || $status == DOWNLOAD_QUEUED || $status == DOWNLOAD_COMPLETE || $status == DOWNLOAD_ACTIVE || $status == DOWNLOAD_READY) {
            list ($done, $queued, $failed) = check_all_dl_done($db, $item);
            if ($queued > 0) { 
                // there are still things to download left
                // possibly a pause interrupted things
                $dlid = $item->get_args();
                update_dlarticle_status($db, $dlid, DOWNLOAD_READY, DOWNLOAD_FINISHED, '<');
                $servers->queue_push($db, $item, FALSE); 
                return;
            } else if ($failed > 0) {
                $item->add_failed_servers($item->get_preferred_server()); // do not try this server again
                // check if there is a server we haven't tried yet
                if ($servers->unused_servers_available($item->get_failed_servers()) !== FALSE) {
                    // we have to try another server
                    update_dlarticle_status($db, $dlid, DOWNLOAD_READY, DOWNLOAD_FAILED, '=');
                    $servers->recreate_download_command($db, $item, FALSE);
                    return;
                } // otherwise simply assume we're complete
            }
            $qstatus = QUEUE_FINISHED;
            update_queue_status($db, $item->get_dbid(), $qstatus, 0, 100);
            echo_debug("All is done (D: $done, Q: $queued, F: $failed)", DEBUG_SERVER);
            echo_debug('download status finished', DEBUG_SERVER);
            $username = $item->get_username();
            $destination = get_download_destination($db, $dlid);

            // Renaming to filename specified in database. Might contain escape characters,
            // but only if an attacker has database access. Check anyway:
            $preview = $item->get_preview();
            if ($done == 0)
                $done_status = DOWNLOAD_FAILED;
            else
                $done_status = DOWNLOAD_COMPLETE;
            update_dlinfo_status ($db, $done_status, $dlid);
            echo_debug("download complete $dlid", DEBUG_SERVER);
            delete_download_article ($db, $dlid, DOWNLOAD_COMPLETE);
            delete_download_article ($db, $dlid, DOWNLOAD_FAILED);
            if ($done > 0) {
                $new_item = new action(NULL, NULL, NULL);
                $new_item->copy($item);
                $item->set_command(COMMAND_DOWNLOAD);

                $username = $item->get_username();
                queue_unpar_unrar($db, $destination, $dlid, $servers, $username, $preview);
            }
        } else if ($status == DOWNLOAD_CANCELLED) {
            $done_status = DOWNLOAD_CANCELLED; // a cancel is permanent
            update_dlinfo_status ($db, $done_status, $dlid);
        } else  {
            echo_debug("Unhandled status of download = $status", DEBUG_SERVER);
        }
    }
}


function check_all_dl_done(DatabaseConnection $db, action $item)
{
    echo_debug_function(DEBUG_MAIN, __FUNCTION__);
    $dlid = $item->get_args();
    $db->escape($dlid, TRUE);
    $sql = "SELECT count(\"ID\") AS \"counter\" FROM downloadarticles WHERE \"downloadID\" = $dlid AND \"status\" < " . DOWNLOAD_FINISHED;
    $queued = $db->execute_query($sql);
    if ($queued === FALSE)
        throw new exception('Database error', ERR_GENERIC_DB_ERROR);
    $sql = "SELECT count(\"ID\") AS \"counter\" FROM downloadarticles WHERE \"downloadID\" = $dlid AND \"status\" = " . DOWNLOAD_FAILED;
    $failed = $db->execute_query($sql);
    if ($failed === FALSE)
        throw new exception('Database error', ERR_GENERIC_DB_ERROR);
    $sql = "SELECT count(\"ID\") AS \"counter\" FROM downloadarticles WHERE \"downloadID\" = $dlid AND \"status\" = " . DOWNLOAD_FINISHED;
    $done = $db->execute_query($sql);
    if ($done === FALSE)
        throw new exception('Database error', ERR_GENERIC_DB_ERROR);
    return array ($done[0]['counter'], $queued[0]['counter'], $failed[0]['counter']);
}


function add_download(DatabaseConnection $db, $username, $unpar, $unrar, $subdl, $delete_files, $status, $destination, $dl_type, $first_run=TRUE)
{
    $db->insert_query('downloadinfo', array('name', 'unpar', 'unrar', 'subdl', 'delete_files', 'status', 'destination', 'username', 'preview', 'size', 'first_run'), 
    array('', $unpar, $unrar, $subdl, $delete_files, $status, $destination, $username, $dl_type, 0, $first_run?1:0));
    $id = $db->get_last_id();
    return $id;
}


function set_download_dir(DatabaseConnection $db, $id, $destination)
{
    assert(is_numeric($id));
    $db->escape($id, TRUE);
    $db->escape($destination, TRUE);
    $sql = "UPDATE downloadinfo SET \"destination\" = $destination WHERE \"ID\" = $id";
    $db->execute_query($sql);
}


function create_download(DatabaseConnection $db, server_data &$servers, $username, $preview = FALSE, $priority=NULL)
{
    global $responses;
    list($code, $dlid, $item_id) = do_create_download($db, $servers, $username, $preview, $priority);
    if ($code == 210) {
        $id_str = "[$item_id] ";
        return sprintf ($responses[210], $dlid, $id_str);
    } else
        return $responses[$code];
}


function do_create_download(DatabaseConnection $db, server_data &$servers, $username, $preview = FALSE, $priority=NULL)
{
    echo_debug_function(DEBUG_SERVER, __FUNCTION__);
    global $responses, $config;
    $status = DOWNLOAD_READY;
    $r = get_pref($db, 'unrar', $username);
    $unrar =  ($r !== FALSE) ? $r : 0; // default off?
    $r = get_pref($db, 'unpar', $username);
    $unpar =  ($r !== FALSE) ? $r : 0; // default off?
    $r = get_pref($db, 'subs_lang', $username, '');
    $subdl =  ($r !== FALSE) ? (($r != '')? 1:0) : 0; // default off?
    $r = get_pref($db, 'delete_files', $username);
    $delete_files =  ($r !== FALSE) ? $r : 0; // default off?
    $id = add_download($db, $username, $unpar, $unrar, $subdl, $delete_files, $status, '', $preview?DLTYPE_PREVIEW:DLTYPE_NORMAL);

    $dl_path_basis = get_dlpath($db);
    $dl_path = find_unique_name($dl_path_basis, TMP_PATH . $username . '/', $id);
    $rv = @create_dir($dl_path, 0775);
    if ($rv === FALSE) {
        write_log("Failed to create directory $dl_path", LOG_ERR);
        return array(405, NULL, NULL);
    }	
    clearstatcache();
    if (!is_writable($dl_path)) {
        write_log("Download directory is not writable: $dl_path", LOG_ERR);
        return array(405, NULL, NULL);
    }
    $id_str = '';
    $item = new action(COMMAND_DOWNLOAD, $id, $username, TRUE);
    $item->set_dlpath($dl_path);
    set_download_dir($db, $id, $dl_path);
    if ($preview)  {
        $item->set_preview(TRUE);
        $priority = 2;
    }
    $res = $servers->queue_push($db, $item, TRUE, QUEUE_BOTTOM, $priority);
    if ($res === FALSE) 
        throw new exception ('Could not queue item', ERR_QUEUE_FAILED);
    inc_dl_lock($db, $id); // lock the dowload so it won't start until it is unlocked and all setdata is added to the article tabels
    if ($preview) {
        update_queue_norestart($db, $res);
    }
    if ($item->is_paused())
        $status = DOWNLOAD_PAUSED;
    else
        $status = DOWNLOAD_QUEUED;
    set_download_destination($db, $id, $dl_path);
    touch ($dl_path . URDD_DOWNLOAD_LOCKFILE); //lock file set to indicate URDD is still doing stuff with it
    update_dlinfo_status ($db, $status, $id);
    return array(210, $id, $item->get_id());
}


function restart_download(server_data &$servers, $username, DatabaseConnection $db, $id, $priority=NULL)
{
    assert(is_numeric($id));
    echo_debug_function(DEBUG_SERVER, __FUNCTION__);
    global $responses, $config;
    $item = new action(COMMAND_DOWNLOAD, $id, $username, FALSE);
    if ($servers->has_equal($item))
        return $responses[406];
    $dl_status = DOWNLOAD_ACTIVE;
    $ready_status = DOWNLOAD_READY;
    update_dlinfo_status ($db, $ready_status, $id, $dl_status);
    if ($item->is_paused()) {
        $status = DOWNLOAD_PAUSED;
        update_dlinfo_status ($db, $status, $id);
    }
    $dl_path = get_download_destination($db, $id);
    $rv = is_dir($dl_path) && is_writable($dl_path);

    if ($rv === FALSE) {
        write_log("Directory is not accessible $dl_path", CONFIG_ERROR);
        return $responses[405];
    }
    $id_str = '';
    echo_debug("re-starting download $id", DEBUG_SERVER);

    $res = $servers->queue_push($db, $item, TRUE, QUEUE_BOTTOM, $priority);
    if ($res === FALSE) 
        throw new exception ('Could not queue item', ERR_QUEUE_FAILED);
    $id_str .= "[{$item->get_id()}] ";
    return sprintf ($responses[210], $id, $id_str);
}


function verify_cksfv(DatabaseConnection $db, $dir, $dlid, pr_list $files, action $item, &$error)
{
    assert(is_numeric($dlid));
    echo_debug_function(DEBUG_SERVER, __FUNCTION__);
    $sfv_cmd = my_escapeshellcmd(get_config_default($db, 'cksfv_path', ''));
    if ($sfv_cmd == '')
        return 'sfv not run';
    $starttime = time();	
    chdir($dir);
    $count = $succ = 0;
    $log_file = my_escapeshellarg($dir . 'sfv.log');
    if (!is_array($files->file_list)){
        write_log("No files found for download $dlid");
        return 'No files found';
    }
    $niceval = get_config($db, 'nice_value');
    if (!is_numeric($niceval) || $niceval < 0 || $niceval > 19) 
        $niceval = 0;
    foreach($files->file_list as $f) {
        if ($f->ext == SFV_EXT) {
            $filename = my_escapeshellarg($f->files[0]);
            exec ("nice -$niceval $sfv_cmd -f ./$filename -q -i >>$log_file 2>&1", $output, $rv);
            $count++;
            if ($rv == 0) 
                $succ++;
        }
    }
    if ($count == $succ) {
        write_log("Successful download $dlid");
        $comment = 'Cksfv complete ';
        $endtime = time();
        $t_time = $endtime - $starttime;
        $status = QUEUE_RUNNING;
        update_queue_status($db, $item->get_dbid(), NULL, $t_time, 50, $comment);
    } else {
        write_log("Incomplete download $dlid");
        update_dlinfo_status($db, DOWNLOAD_CKSFV_FAILED, $dlid);
        $dl_status = DOWNLOAD_CKSFV_FAILED;

        $error = TRUE;
        $status = QUEUE_FAILED;
        $comment = 'Cksfv failed ';
    }
    return $comment;
}


function verify_par(DatabaseConnection $db, $dir, $dlid, pr_list $files, action $item, &$error, &$unpar)
{
    assert(is_numeric($dlid));
    echo_debug_function(DEBUG_SERVER, __FUNCTION__);
    $par_cmd = my_escapeshellcmd(get_config_default($db, 'unpar_path', ''));
    if ($par_cmd === '') {
        $unpar = FALSE;
        return 'Par not run';
    }
    $starttime = time();	
    chdir($dir);
    $count = $succ = 0;
    $par2_params = my_escapeshellarg(get_config($db, 'unpar_pars'), FALSE);
    $log_file = my_escapeshellarg($dir . 'par2.log');
    $niceval = get_config($db, 'nice_value');
    if (!is_numeric($niceval) || $niceval < 0 || $niceval > 19) 
        $niceval = 0;
    foreach($files->file_list as $f) {
        if ($f->ext == PAR_EXT) {
            $filename = my_escapeshellarg($f->files[0]);
            exec ("nice -$niceval $par_cmd $par2_params ./$filename >>$log_file 2>&1", $output, $rv);
            $count++;
            if ($rv == 0) 
                $succ++;
        }
    }
    if ($count == 0) {
        $unpar = 0;
        return 'No par files';
    }
    if ($count == $succ) {
        write_log("Successful download $dlid");
        $comment = 'PAR2 complete ';
        $endtime = time();
        $t_time = $endtime - $starttime;
        $status = QUEUE_RUNNING;
        $error = FALSE;
        update_queue_status($db, $item->get_dbid(), NULL, $t_time, 50, $comment);
    } else {
        write_log("Incomplete download $dlid");
        update_dlinfo_status($db, DOWNLOAD_PAR_FAILED, $dlid);
        $dl_status = DOWNLOAD_PAR_FAILED;
        $status = QUEUE_FAILED;
        $comment = 'PAR2 failed ';
        $unrar = 0; // we don't need to unrar if it failed
        $error = TRUE;
    }
    return $comment;
}


function decompress(DatabaseConnection $db, $type, $dir, pr_list $files, $password, $dlid, &$dl_status, &$error)
{
    assert(is_numeric($dlid));
    echo_debug_function(DEBUG_SERVER, __FUNCTION__);
    $rar_cmd = get_config_default($db, 'unrar_path', '');
    $arj_cmd = get_config_default($db, 'unarj_path', '');
    $zip_cmd = get_config_default($db, 'unzip_path', '');
    $ace_cmd = get_config_default($db, 'unace_path', '');
    $zr7_cmd = get_config_default($db, 'un7zr_path', '');

    $rar_arg = get_config_default($db, 'unrar_pars', '');
    $arj_arg = get_config_default($db, 'unarj_pars', '');
    $zip_arg = get_config_default($db, 'unzip_pars', '');
    $ace_arg = get_config_default($db, 'unace_pars', '');
    $zr7_arg = get_config_default($db, 'un7zr_pars', '');

    $archive_types = array ( // commnad, options, password option, nopassword
        //RAR_EXT => array($rar_cmd, 'x -y -kb -o- -inul -c-', '-p@@@', '-p-'),
        RAR_EXT => array($rar_cmd, $rar_arg, '-p@@@', '-p-'),
        ACE_EXT => array($ace_cmd, $ace_arg, '-p@@@', ''),
        ZIP_EXT => array($zip_cmd, $zip_arg, '-P @@@', ''),
        ARJ_EXT => array($arj_cmd, $arj_arg, '-g@@@', ''),
        ZR7_EXT => array($zr7_cmd, $zr7_arg, '-p@@@', '')
    );

    chdir($dir);
    $niceval = get_config($db, 'nice_value');
    if (!is_numeric($niceval) || $niceval < 0 || $niceval > 19) 
        $niceval = 0;
    $count = $succ = 0;
    $comment = '';
    $log_file = my_escapeshellarg($dir . "$type.log");
    foreach($files->file_list as $f) {
        if ($f->ext == $type) {
            $count++;
            foreach($f->files as $filename) {
                echo_debug("Trying to expand $filename", DEBUG_SERVER);
                $filename = my_escapeshellarg($filename);
                $cmd = my_escapeshellcmd($archive_types[$type][0]);
                if ($cmd === '')
                    continue;
                $options = my_escapeshellarg($archive_types[$type][1], FALSE);

                if ($password != '') {
                    $pw_opt = $archive_types[$type][2];
                    $pw_opt = my_escapeshellarg(str_replace('@@@', $password, $pw_opt));
                } else
                    $pw_opt = $archive_types[$type][3];
                $cmd_line = "nice -$niceval $cmd $options $pw_opt ./$filename >>$log_file 2>&1"; 

                exec ($cmd_line, $output, $rv); 
                if ($rv == 0) {
                    $succ++;
                    break;
                } else if ($type != ZR7_EXT)  // only 7z is picky about the order?? RAR is not at least; ARJ, ACE, ZIP are not tested
                    break;
                else 
                    echo_debug("'$filename': Error code $rv", DEBUG_SERVER);
            }
        }
    }
    if ($count == 0) {
        echo_debug("no $type files found", DEBUG_SERVER);
    } else if ($count == $succ) {
        write_log("Successfully decompressed download $dlid ($count archives)");
        $comment .= "Decompression successful ($count archives) ";
    }
    else {
        $comment .= " Decompression failed ($count archives found; $succ decompressed)";
        update_dlinfo_status($db, DOWNLOAD_RAR_FAILED, $dlid);
        $dl_status = DOWNLOAD_RAR_FAILED;
        write_log("Decompressing download $dlid failed ($count archives found; $succ decompressed) ");
        $error = TRUE;
    }
    return $comment;
}


function concat_files(DatabaseConnection $db, pr_list $files, $dlid, $dir, &$error, &$dl_status)
{
    assert(is_numeric($dlid));
    echo_debug_function(DEBUG_SERVER, __FUNCTION__);
    $comment = '';
    foreach($files->file_list as $f) {
        if ($f->ext == CAT_EXT) {
            natcasesort($f->files);
            $filename = $f->files[0];
            $filename = preg_replace("/\.\d{3}$/", '', $filename);
            $fp = fopen ($dir . $filename, "x");
            if ($fp === FALSE) {
                $comment .= ' could not create file';
                update_dlinfo_status($db, DOWNLOAD_RAR_FAILED, $dlid);
                $dl_status = DOWNLOAD_RAR_FAILED;
                write_log("Concatenating download $dlid failed");
                $error = TRUE;
            }
            foreach ($f->files as $file) {
                $fpr = fopen ($dir . $file, 'r');
                if ($fpr === FALSE) {
                    $comment .= " could not create file $file";
                    update_dlinfo_status($db, DOWNLOAD_RAR_FAILED, $dlid);
                    $dl_status = DOWNLOAD_RAR_FAILED;
                    write_log("Concatenating download $dlid failed");
                    $error = TRUE;
                    break;
                }
                while (!feof($fpr)) {
                    $rv = @fwrite($fp, fread($fpr, 8192));
                    if ($rv === FALSE) {
                        $comment .= ' could not write file';
                        update_dlinfo_status($db, DOWNLOAD_RAR_FAILED, $dlid);
                        $dl_status = DOWNLOAD_RAR_FAILED;
                        write_log("Concatenating download $dlid failed");
                        $error = TRUE;
                        break;
                    }
                }	
                fclose ($fpr);
            }
            fclose ($fp);
        }
    }
    return $comment;
}


function uudecode(DatabaseConnection $db, pr_list $files, $dlid, $dir, &$error, &$dl_status, &$counter)
{
    assert(is_numeric($dlid));
    echo_debug_function(DEBUG_SERVER, __FUNCTION__);
    $comment = '';
    $counter++;
    static $descriptorspec = array( 
        0 => array('pipe', 'r'), // where we will write to
        //1 => array('file', '/tmp/out1', 'a'), // we don't want the output
        //2 => array('file', '/tmp/out2', 'a') // or the errors
        1 => array('file', '/dev/null', 'w'), // we don't want the output
        2 => array('file', '/dev/null', 'w') // or the errors
    );
    $yydecode = my_escapeshellcmd(get_config_default($db, 'yydecode_path', ''));
    $yydecode_pars = my_escapeshellarg(get_config_default($db, 'yydecode_pars', ''), FALSE);
    if ($yydecode == '')
        throw new exception('yydecode not found', ERR_CONFIG_ERROR);

    $cmd = "/bin/sh -c '$yydecode $yydecode_pars ' ";
    $pipes = array();
    chdir($dir);
    $process = proc_open($cmd, $descriptorspec, $pipes, $dir, NULL, array('binary_pipes'));
    if ($process === FALSE) {
        update_dlinfo_status($db, DOWNLOAD_ERROR, $dlid);
        write_log('Failed to open pipe', LOG_ERR);
        $error = TRUE;
        $dl_status = DOWNLOAD_ERROR;
        return 'Failed to open pipe'; 
    }
    foreach($files->file_list as $f) {
        if ($f->ext == UUE_EXT) {
            $counter++;
            natcasesort($f->files);
            foreach($f->files as $f) {
                $contents = file_get_contents($dir . $f);
                fwrite($pipes[0], $contents);
            }
        }
    }
    pclose($pipes[0]);
    proc_close($process);
    return $comment;
}


function create_download_threads(server_data &$servers, DatabaseConnection $db, action $item)
{
    echo_debug_function(DEBUG_MAIN, __FUNCTION__);
    $dlid = $item->get_args();
    try {
        if (check_dl_lock($db, $dlid) === FALSE) { // if db still locked
            schedule_locked_item($db, $servers, $item);
            return;
        }
        $srv_id = $servers->find_free_slot($item->get_all_failed_servers(), $item->need_posting()); // is there a server that has a free slot
        if ($item->get_preview()) {
            $nr_threads = 1;
            $priority = 1; // preview always gets ahead of anything
        } else {
            if ($srv_id === FALSE) {
                write_log('Server ID not specified', LOG_ERR);
                throw new exception('Server ID not specified');
            }
            $max_dl_nntp = get_config($db, 'nntp_maxdlthreads');
            $nr_threads = $servers->get_max_threads($srv_id);
            if ($max_dl_nntp > 0) 
                $nr_threads = min ($max_dl_nntp, $nr_threads);
            $priority = 2; // a ready download gets the highest priority so it is scheduled in asap; only previews will overrule this
        }
        $dlsize = get_download_size($db, $dlid);
        // we want to limit the number of threads started to avoid unneeded threads
        if ($dlsize <= (2 * 1024 * 1024))
            $nr_threads = 1;
        elseif ($dlsize <= (5 * 1024 * 1024))
            $nr_threads = min($nr_threads, 2);
        elseif ($dlsize <= (10 * 1024 * 1024))
            $nr_threads = min($nr_threads, 3);
        elseif ($dlsize <= (20 * 1024 * 1024))
            $nr_threads = min($nr_threads, 5);
        for ($i = 0 ; $i < $nr_threads; $i++)	{
            $new_item = new action(NULL, NULL, NULL); // create a dummy...
            $new_item->copy($item); // fill it with data here
            $new_item->set_command(COMMAND_DOWNLOAD_ACTION);
            $new_item->set_need_nntp(TRUE);
            $new_item->set_priority($priority, 'root'); 
            $res = $servers->queue_push($db, $new_item, TRUE, QUEUE_BOTTOM,  NULL);
            if ($item->get_preview())
                update_queue_norestart($db, $res);
        }
    } catch (exception $e) {
        write_log("Cannot find download $dlid", LOG_ERR);
    }

    try {
        $servers->queue_delete($db, $item->get_id(), 'root', TRUE); // remove the item from the queue
        update_queue_status($db, $item->get_dbid(), QUEUE_REMOVED, 0, 100, '');
    } catch (exception $e) { 
        write_log($e->getMessage(), LOG_ERR); 
        throw $e;
    }
    return;
}


function create_post_threads(server_data &$servers, DatabaseConnection $db, action $item)
{
    echo_debug_function(DEBUG_MAIN, __FUNCTION__);
    $dlid = $item->get_args();
    try {
     /*   if (check_dl_lock($db, $dlid) === FALSE) { // if db still locked
            schedule_locked_item($db, $servers, $item);
            return;
     }*/
        $srv_id = $servers->find_free_slot($item->get_all_failed_servers(), $item->need_posting()); // is there a server that has a free slot
        if ($item->get_preview()) {
            $nr_threads = 1;
            $priority = 1; // preview always gets ahead of anything
        } else {
            if ($srv_id === FALSE) {
                write_log('Server ID not specified', LOG_ERR);
                throw new exception('Server ID not specified');
            }
            $max_dl_nntp = get_config($db, 'nntp_maxdlthreads');
            $nr_threads = $servers->get_max_threads($srv_id);
            if ($max_dl_nntp > 0) 
                $nr_threads = min ($max_dl_nntp, $nr_threads);
            $priority = 2; // a ready download gets the highest priority so it is scheduled in asap; only previews will overrule this
        }
        for ($i = 0 ; $i < $nr_threads; $i++)	{
            $new_item = new action(NULL, NULL, NULL); // create a dummy...
            $new_item->copy($item); // fill it with data here
            $new_item->set_command(COMMAND_POST_ACTION);
            $new_item->set_need_nntp(TRUE);
            $new_item->set_priority($priority, 'root'); 
            $res = $servers->queue_push($db, $new_item, TRUE, QUEUE_BOTTOM,  NULL);
        }
    } catch (exception $e) {
        write_log("Cannot find post $dlid", LOG_ERR);
    }

    try {
        $servers->queue_delete($db, $item->get_id(), 'root', TRUE); // remove the item from the queue
        update_queue_status($db, $item->get_dbid(), QUEUE_REMOVED, 0, 100, '');
    } catch (exception $e) { 
        write_log($e->getMessage(), LOG_ERR); 
        throw $e;
    }
    return;
}


function run_scripts(DatabaseConnection$db, action $item, $dlid, $dl_status, $global = FALSE)
{ // global == TRUE run admin set scripts; global == FALSE run user set scripts
    $destination = get_download_destination($db, $dlid);
    $dlpath = my_escapeshellcmd($destination);
    $dlid = my_escapeshellcmd($dlid);
    $dl_status = my_escapeshellcmd($dl_status);
    $rv = $rv_tmp = 0;

    $username = $item->get_username();
    $scripts_path =  get_dlpath($db);
    $scripts_path .= SCRIPTS_PATH;
    if ($global === TRUE) {
        $scripts = get_pref($db, 'global_scripts', $username);
    } else {
        $scripts_path .= $username . '/';
        $scripts = get_pref($db, 'user_scripts', $username);
    }
    $scripts = explode("\n", $scripts);
    sort($scripts);
    foreach ($scripts as $script) {
        if ($script == '') 
            continue;
        if (verify_script($db, $scripts_path, $script) == '') {
            unset($output);
            $cmd = $scripts_path . $script . " $dlpath $dlid $dl_status";
            write_log("Running script $script for download $dlid", LOG_NOTICE);
            exec($cmd, $output, $rv_tmp);
            if ($rv_tmp != 0)
                write_log("The script $script exited with error code $rv", LOG_ERR);
            if ($rv == 0)
                $rv = $rv_tmp;
        }
    }
    return $rv;
}


function run_all_scripts(DatabaseConnection$db, action $item, $dlid, $dl_status)
{
    echo_debug_function(DEBUG_SERVER, __FUNCTION__);
    $global_scripts = get_config($db, 'allow_global_scripts');
    $user_scripts = get_config($db, 'allow_user_scripts');
    if ($global_scripts == 0)  // not allowed to run any scripts
        return;

    run_scripts($db, $item, $dlid, $dl_status, TRUE);

    if ($user_scripts == 0) // not allowed to run scripts from a user
        return;

    run_scripts($db, $item, $dlid, $dl_status, FALSE);
}


function move_sub_files($from, $to)
{
    // Move all subtitle files somewhere:
    $sub_ext = array('.srt', '.SRT', '.sub', '.SUB', '.idx', '.IDX');
    foreach($sub_ext as $ext) {

	// Move files to the $to folder:
        foreach (glob("$from*$ext", GLOB_NOSORT) as $file) {
            rename($file, $to . basename($file));
        }

    }
}


function rename_sub_files($folder, $language, $destination)
{
    // Rename all subtitle files:
    $sub_ext = array('.srt', '.SRT', '.sub', '.SUB', '.idx', '.IDX');
    foreach($sub_ext as $ext) {

	// Change 'Movie.srt' to 'Movie.en.srt':
        foreach (glob("$folder*$ext", GLOB_NOSORT) as $file) {
            $tofile = $destination . '/' . basename($file, $ext) . '.' . $language . $ext;
	    rename($file, $tofile);
            write_log("Subtitle found: " . basename($tofile), LOG_NOTICE);
        }

    }
}


function download_subs(DatabaseConnection$db, action $item, $dlid, $dir, $username)
{
    // $dir : /somewhere/urd/tmp/User1/1/
    echo_debug_function(DEBUG_SERVER, __FUNCTION__);

    // Variables:
    $originalsubs = 'original_subtitles';
    $addedsubs = 'downloaded_subtitles';
    $sublog = 'subs.log';

    $lang = get_pref($db, 'subs_lang', $username, '');
    if ($lang == '') 
        return 'subdl not run';
    $langs = explode(',', $lang);

    $subdownloader_cmd = my_escapeshellcmd(get_config_default($db, 'subdownloader_path', ''));
    $subdownloader_pars = my_escapeshellarg(get_config_default($db, 'subdownloader_pars', ''), FALSE);
    if ($subdownloader_cmd === '') {
        $unpar = FALSE;
        return 'subdl not run';
    }

    // Store original subtitles in the 'originalsubs' folder:
    $target_dir = $dir . $originalsubs;
    $target_dir = find_unique_name($target_dir, '', '', '', FALSE);
    create_dir($target_dir, 0775);
    move_sub_files($dir, $target_dir);
    @rmdir($target_dir); // Only works on empty folders

    // Put all subtitles in the addedsubs directory later on
    create_dir($dir . $addedsubs, 0775);

    $log_file = my_escapeshellarg($dir . $sublog);
    foreach($langs as $l) {
        $l = trim($l);
        write_log("Getting subs for $l", LOG_NOTICE);
        $cmd = "/bin/sh -c '$subdownloader_cmd $subdownloader_pars --lang=$l 2>> $log_file >>$log_file'";
        exec($cmd, $foo, $rc);
        // todo do sth with the rc value :D

	// Rename subtitle files so they include the language in the filename, prevents collisions when
	// multiple subtitles are downloaded because multiple languages are selected.
	rename_sub_files($dir, $l, $dir . $addedsubs);
	// (File sub.srt for language 'en' is now moved to 'x/downloaded_subtitles/sub.en.srt)
    }

    // Now move all subtitles back and remove the downloaded_subtitles folder
    move_sub_files($dir . $addedsubs, $dir);
    @rmdir($dir . $addedsubs);
}

?>
